package com.study.feature.extract

import org.apache.spark.ml.feature.FeatureHasher
import org.apache.spark.sql.SparkSession

/**
 * 特征提取-FeatureHasher
 *
 * 特征哈希是将一组类别特征或数值特征投影到指定维度的特征向量中(通常显著小于原始特征空间的维度)。
 * 这是通过哈希技巧将特征映射到特征向量中的索引来完成的。
 *
 * @author stephen
 * @date 2019-08-28 11:08
 */
object FeatureHasherDemo {

  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName(this.getClass.getSimpleName)
      .master("local[*]")
      .getOrCreate()

    spark.sparkContext.setLogLevel("warn")

    val dataset = spark.createDataFrame(Seq(
      (2.2, true, "1", "foo"),
      (3.3, false, "2", "bar"),
      (4.4, false, "3", "baz"),
      (5.5, false, "4", "foo")
    )).toDF("real", "bool", "stringNum", "string")

    val hasher = new FeatureHasher()
      .setInputCols("real", "bool", "stringNum", "string")
      .setOutputCol("features")

    val featurized = hasher.transform(dataset)
    featurized.show(false)
  }
}
